#!/bin/bash
#
# good for job less than 30 minutes
# have to pass bank using -A or set PROJECT environment variable

export FROM_CHARMRUN='1'

args=""
pes=1
machinefile=""
pernode=32
envs=""
QUIET=0

while [ $# -gt 0 ]
do
	case $1 in
	+ppn)
		args=$args" +ppn "$2
		shift
		;;
	+ppn*)
		args=$args" "$1
		;;
	+p)
		pes=$2
		shift
		;;
	+p[0-9]*)
		pes=`echo $1 | awk '{print substr($1,3)}'`
		;;
  -machinefile)
		machinefile=$2
		args=" "$1" "$2" "$args
		shift
		;;
	--mode)
		args=" "$1" "$2" "$args
		shift
		;;
	-A)
		args=" "$1" "$2" "$args
		shift
		;;
	--envs)
		envs=$envs" --envs " $2
		shift
		;;
	++quiet)
		QUIET=1
		;;
	++no-quiet)
		QUIET=0
		;;
  --pernode)
    pernode=$2
    shift
    ;;
	*) 
		args=$args" "$1
		;;
	esac
	shift
done


test $QUIET -eq 1 && args=$args" "++quiet

test $QUIET -eq 0 && printf "\nRunning on $pes processors: $args\n"

if test -n "$COBALT_JOBID"
then
  # charmrun called from script
  test $QUIET -eq 0 && echo "runjob -p $pernode -n $pes --block $COBALT_PARTNAME --envs BG_SHAREDMEMSIZE=32MB $envs : $args"
  runjob -p $pernode -n $pes --block $COBALT_PARTNAME --envs BG_SHAREDMEMSIZE=32MB $envs : $args
  exit $?
else

queue_stat=qstat
queue_kill=qdel
queue_sub=qsub

while [ true ]
do

test $QUIET -eq 0 && echo "Submitting batch job for> $pes $args"
test $QUIET -eq 0 && echo " using the command> $queue_sub -t 30 -n $pes  $args"
jobid=""
while [ -z "$jobid" ]
do
  jobid=`$queue_sub -t 30 -n $pes  $args 2>err.$$ |tail -1`
  if grep 'not found' err.$$ > /dev/null
  then
    cat err.$$
    rm -f err.$$
    exit 1
  fi
  sleep 10
done
test $QUIET -eq 0 && echo "Job enqueued under job ID $jobid"

output=$jobid.output
err=$jobid.error
log=$jobid.cobaltlog

End() {
        echo "Charmrun> $queue_kill $jobid ..."
        $queue_kill $jobid
        rm -f $script
        exit $1
}

# kill job if interrupted
trap 'End 1' 2 3

# Wait for the job to complete, by checking its status
while [ true ]
do
        $queue_stat $jobid >>tmp.$$
	      exitstatus=$?

        if test $exitstatus -ne 0
        then
            rm -f tmp.$$
            # job not in the queue now
            status=`grep -c "exited with status 0" $err`
		        if test $status != 0 ;
            then
              test $QUIET -eq 0 && echo "job exited with status 0: output in $output"
              exit 0
            else
              echo "job did not exit with status 0; error in $err, output in $output"
              # display the error, output and cobaltlog for failed jobs
              echo "==============================================================="
              echo "Job error: $err"
              cat $err
              echo "==============================================================="
              echo "Job output: $output"
              cat $output
              echo "==============================================================="
              echo "Job scheduler log: $log"
              cat $log
              echo "==============================================================="
              exit 1
            fi
        fi
	      sleep 20
done
done

fi
